#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#		to be called from test21.30
#		Tests if the GrpCpus limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test GrpCpus
proc inc21_30_2 { wait_reason } {
	global srun salloc acct bin_sleep jobmatch job_id1 job_id2 number
	global grcpu_num test_node

	set job_id1 0
	set job_id2 0

	log_info "Starting GrpCPUs limit test"

	# Raise an error to abort the catch block
	set exception_code [catch {

		set jobmatch 0
		spawn $salloc --account=$acct -w$test_node -t1 -n$grcpu_num $srun $bin_sleep 5
		expect {
			-re "Granted job allocation ($number)" {
				set job_id1 $expect_out(1,string)
				log_debug "Job $job_id1 has been submitted"
				incr jobmatch
			}
			-re "Pending job allocation ($number)" {
				set job_id1 $expect_out(1,string)
				error "Job should be running, but is not. If you have CR_CORE_* and have ThreadsPerCore > 1 this could happen"
			}
			-re "error" {
				error "Job allocation should not have failed"
			}
			timeout {
				error "salloc is not reponding"
			}
			eof {
				wait
			}
		}

		spawn $salloc --account=$acct -w$test_node -t1 -n[expr $grcpu_num + 1] $srun $bin_sleep 5
		expect {
			-re "Granted job allocation ($number)" {
				set job_id2 $expect_out(1,string)
				error "Job should be pending, but is not"
			}
			-re "Pending job allocation ($number)" {
				set job_id2 $expect_out(1,string)
				log_debug "Job $job_id2 is waiting for resources. This is expected"
				incr jobmatch
			}
			timeout {
				error "salloc is not reponding If you have CR_CORE_* and have ThreadsPerCore > 1 this could happen."
			}
			eof {
				wait
			}
		}
		if {$jobmatch != 2} {
			error "jobs were not submitted"
		}

		# Checks the state of the job
		subtest [check_job_state $job_id2 PENDING] "Job should be in the PENDING state"
		subtest [check_reason $job_id2 $wait_reason] "Job should have wait reason $wait_reason"

	} message] ; # Store the error message in $message

	# Cancels remaining jobs
	cancel_job $job_id1
	cancel_job $job_id2

	# Convert any errors into failures (after cleaning up)
	if {$exception_code == 1} { ; # errors only
		fail "Failure testing $wait_reason: $message"
	}
}
